__author__ = "Henry Heberle"
__date__ = "$25/10/2010 20:34:29$"

from decimal import *
import random
import sys
from time import gmtime
from time import strftime
import zipfile

import matplotlib.pyplot as plt
import nltk
from nltk import stem
from nltk.corpus import stopwords
from nltk.probability import *
import vtk
from vtk import *

if __name__ == "__main__":

    punctuations = [']', '[', '.', ',', ';', '"', '\'', '?', '(', ')', ':', '-',
    '_', '`', "#", "&", "|", "--", "&quot", "&ndash", "&mdash",
    "/prnewswire-usnewswire/", "'s", "u'"]

    stopwords = nltk.corpus.stopwords.words('english')
    notwords = punctuations + stopwords
    #print "stopwords:"
    #print stopwords

    tempo = []
    #tempo.append(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())


    def similarity(I, J):
        if I == J:
            return 0.5
        try:
            A = len(I | J)   #c<=5 -> A/2B <= 5 -> A/B <= 10 -> 100/10 ou 50/5 ou 25/2,5 ou 10/1
            B = len(I & J)
            #print "Tamanho: "+ str(A) +"/"+str(B)
            co = A / (B * 2)
        except ZeroDivisionError:
            return -1
        return co

    all_words = []
    fwc = {} #fdist_words_category

    #Leitura do feed de noticias
    #Transforma cada noticia em uma lista de palavras
    #Armazena a lista i em words_text[i]
    words_text = []
    words_text_list = []
    titleList = []
    summaryList = []
    classList = []
    classListNumeric = []
    wnl = stem.WordNetLemmatizer()

    zf = zipfile.ZipFile("news.zip", "r")

    limit = int(sys.argv[3])
    limit_aux = 0

    keys_category = []
    
    for name in zf.namelist():
        if limit_aux < limit:
            try:
                data = zf.read(name)
                data.encode("ascii")
                data_splited = data.split("\n")
                title = nltk.clean_html(data_splited[0])
                summary = nltk.clean_html(data_splited[1])
                text = title + " " + summary
                
                category = name[:3]
                if category not in keys_category:
                    keys_category.append(category)
                key = keys_category.index(category)

                #key = get_key(name[:3]) #number of clategory of a new
                #print "::: "+key+" ::: "+text
                #transforma o texto em lista
                words = [w.lower() for w in nltk.word_tokenize(text) if len(w) > 1]

                #trata algumas palavras por conta de unicode e
                #ponto final em final de texto
                for i in range(len(words)):
                    words[i].replace(".", " ")

                #remove palavras com 1 caractere
                for wd in words:
                    if len(wd) < 2:
                        words.remove(wd)

                #remove stopwords
                words = [w.lower() for w in words if w not in notwords]

                if sys.argv[1] == "porter":
                    #Stemmers
                    porter = nltk.PorterStemmer()
                    porterWords = [porter.stem(w) for w in words]
                    processedWords = porterWords
                elif sys.argv[1] == "lancaster":
                    lancaster = nltk.LancasterStemmer()
                    lancasterWords = [lancaster.stem(w) for w in words]
                    processedWords = lancasterWords
                elif sys.argv[1] == "lemmatizer":
                    #lemas. ex: loves se torna: love
                    lemmatizedWords = [wnl.lemmatize(w) for w in words]
                    processedWords = lemmatizedWords

                #elimina palavras repetidas e cria um conjunto para ser armazenado
                setText = set(processedWords)
                #listSet = list(setText)

                #print text
                #print listSet
                #print lemmatizedWords
                #raw_input()

                words_text.append(setText) #lista de conjuntos de palavras
                words_text_list.append(processedWords) #lista de listas de palavras (palavras se repetem)

                #listas de titulos e resumos das noticias, mapeada para cada vertice i
                titleList.append(title)
                summaryList.append(summary)

                #listas de categorias, mapeadas para cada vertice i
                classList.append(category) #string da categoria
                classListNumeric.append(key) #numero  da categoria
                limit_aux += 1
            except Exception as e:
                print e
                pass
    zf.close()

    #Diminui o espaco de palavras
    #words_text = words_text[:300] #mudar no SQL

    weight = []
    number_of_texts = len(words_text)    

    def getConstPerCent(percent):
        """Retorna uma constante que representa a distancia entre as noticias;
                              quao diferentes sao. 0.5 significa muito similar"""
        return 100 / (2 * percent)
    #similarity: % -> Const
    #100 = 0.50		40 = 1.25
    #90  = 0.56		30 = 1.667
    #80  = 0.63		20 = 2.5
    #70  = 0.72		10 = 5.0
    #60  = 0.84		05 = 10.0
    #50  = 1.00

    edgesListaux = [] #list of edges. some of them will be removed
    weightListaux = []
    max = 99999999

    constante = getConstPerCent(15)

    """
     0 1 2 3 4
    0  x x x x   i = 0 j = 1,2,3,4
    1    x x x   i = 1 j = 2,3,4
    2      x x   i = 2 j = 3,4
    3        x   i = 3 j = 4
    4

    """
    for i in range(0, number_of_texts-1):
        l = []
        for j in range(i+1, number_of_texts):
            co = similarity(words_text[i], words_text[j])
            if co == -1:
                co = max
            l.append(co)
            if co < constante:                
                edgesListaux.append([i, j])
                weightListaux.append(co)
        weight.append(l)

    print "Calculou pesos para as arestas, geral, do grafo"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    def printNews(i, j):
        print titleList[i] + "\n" + summaryList[i] + "\n"
        print titleList[j] + "\n" + summaryList[j]
        print "--------------------------------------------------------------"

    def getProbability(edgs):
        used = []
        prob = []
        for v in edgs:
            i = v[0]
            j = v[1]
            if i != j and [i, j] not in used and [j, i] not in used:
                used.append([i, j])
                printNews(i, j)
                c = similarity(words_text[i], words_text[j])
                simil = raw_input()
                prob.append([[i, j], c, simil])
        return prob

    def saveProb(prob):
        file = open("prob.txt", "w")
        for p in prob:
            edg = p[0]
            c = p[1]
            simil = p[2]
            line = str(edg[0]) + " " + str(edg[1]) + " " + str(c) + " " + str(simil) + "\n"
            file.write(line)
        file.close()

    def selectEdges(num):
        edg1, edg2, edg3, edg4 = [], [], [], []
        i = 0
        for l in weight:
            j = 0
            for c in l:
                if c < 1.0 and c >= 0.5:
                    edg1.append([i, j])
                elif c < 1.5 and c >= 1.0:
                    edg2.append([i, j])
                elif c < 2.0 and c >= 1.5:
                    edg3.append([i, j])
                elif c < 2.5 and c >= 2.0:
                    edg4.append([i, j])
                j += 1
            i += 1
        random.shuffle(edg1)
        random.shuffle(edg2)
        random.shuffle(edg4)
        random.shuffle(edg1)
        random.shuffle(edg3)
        random.shuffle(edg3)
        random.shuffle(edg4)
        random.shuffle(edg2)
        print edg1
        print edg2
        print edg3
        print edg4
        edg = edg1[:num] + edg2[:num] + edg3[:num] + edg4[:num]
        return edg

    #selectedEdges  = selectEdges(3)
    #print selectedEdges
    #prob = getProbability(selectedEdges)
    #saveProb(prob)
    """
    x, y = [], []
    for obj in prob:
        x.append(obj[1])
        y.append(obj[2])
    plt.plot(x, y)
    plt.axis([0, 3, 0, 4])
    plt.show()
    """
    vertex = []
    g = vtk.vtkMutableDirectedGraph()
    vertices = vtk.vtkDoubleArray()
    vertices.SetNumberOfComponents(1)
    vertices.SetName("Vertices")

    #Create N vertices
    for k in range(number_of_texts):
        vertex.append(g.AddVertex())
        vertices.InsertNextValue(k)


    colors_v = vtk.vtkDoubleArray()
    colors_v.SetNumberOfComponents(1)
    colors_v.SetName("color")
    #colors for N vertices
    for k in range(number_of_texts):
        colors_v.InsertNextValue(classListNumeric[k])

    #Colors for edges that will be created
    colors_e = vtk.vtkDoubleArray()
    colors_e.SetNumberOfComponents(1)
    colors_e.SetName("color")

    total = 0
    #auxiliar lists
    weightList = []

    #weights of edges of the vtkGraph
    weights = vtk.vtkDoubleArray()
    weights.SetNumberOfComponents(1)
    weights.SetName("Weights")

    """
    i = 0
    constante = getConstPerCent(5)
    #Select only some edges to create. Big distances(diferent texts) will not be created.
    for l in weight:
        j = 0
        for c in l:
            if c < constante:
                if i != j and [j, i] not in edgesListaux and [i, j] not in edgesListaux:
                    a = [i, j]
                    edgesListaux.append(a)
                    weightListaux.append(c)
            j += 1
        i += 1
     """

    #auxiliar list to treate (i,j), (i,i) and (j,i) edges like one.
    edgesList = []

    #number of edges created in the last for-loop
    motivoEdge = []
    porsozinho = 0
    porvizinho = 0
    porotimo = 0
    portextocurto = 0

    #return a color value for a weight gived, const
    def getColorEdge(weightE):
        if weightE < 1.25:
            return 5
        elif weightE <= 1.60:
            return 4
        elif weightE <= 2.19:
            return 3
        elif weightE <= 2.7:
            return 2
        elif weightE <= 5.0:
            return 1
        else:
            return 0

    #edges that will really be considered to be drawn
    resultEdges = []


    #Euristics to draw the graph
    for k in range(len(edgesListaux)):
        edge = edgesListaux[k]
        v0 = edge[0]
        v1 = edge[1]        
        C = getConstPerCent(40)
        if weightListaux[k] >= C: #se os textos forem distantes, nao importa o tamanho deles
            Co = getConstPerCent(1)


            #Pode ser melhorado armazenando uma lista de vertices usados para formar
            #esta de arestas no loop for acima, que gera a matriz de adjacencia geral
            
            #seleciona vertices que possuem arestas, ou seja, cujo peso eh maior que uma constante C
            #de um loop-for anterior
            vList0 = [edg[1] for edg in edgesListaux if edg[0] == v0] #vertices vizinhos
            vList0 = vList0 + [edg[0] for edg in edgesListaux if edg[1] == v0] #vertices vizinhos
            #vList0 = list(set(vList0))

            vList1 = [edg[1] for edg in edgesListaux if edg[0] == v1]
            vList1 = vList1 + [edg[0] for edg in edgesListaux if edg[1] == v1]
            #vList1 = list(set(vList1))

            text0 = words_text[v0]
            for v in vList0:
                text0 = text0 | words_text[v] #lembrete: texto de indice v eh representado pelo vertice de numero v

            text1 = words_text[v1]
            for v in vList1:
                text1 = text1 | words_text[v]


            #calcular a similaridade entre o grupo de vizinhos de cada vertice e saber se a
            #aresta deve ou nao ser desenhada
            if len(words_text[v0]) > 15 and len(words_text[v1]) > 15:
                const = similarity(text0, text1)
                Cgrupo = getConstPerCent(25)
                #print "entrou em vizinhanca!"
                if const <= Cgrupo:
                    #print "similaridade de grupo boa, adiciona aresta"
                    g.AddGraphEdge(v0, v1)
                    weights.InsertNextValue(weightListaux[k] / 1.0)
                    weightList.append(weightListaux[k] / 1.0)
                    colors_e.InsertNextValue(getColorEdge(weightListaux[k]))
                    edgesList.append([v0, v1])
                    motivoEdge.append("similaridade de grupo, vizinhos")
                    porvizinho += 1
        #se o peso da aresta for pequeno, nao consideramos os vizinhos
        #para peso pequeno o tamanho do texto importa
        else:
            C2 = getConstPerCent(60)
            if(weightListaux[k] > C2):
                if len(words_text[v0]) < 20 and len(words_text[v1]) < 20:
                    print "textos potencialmente ruins"
                else:
                    #print "boa precisao! adicionou direto"
                    g.AddGraphEdge(v0, v1)
                    weights.InsertNextValue(weightListaux[k] / 1.0)
                    weightList.append(weightListaux[k] / 1.0)
                    colors_e.InsertNextValue(getColorEdge(weightListaux[k]))
                    edgesList.append([v0, v1])
                    motivoEdge.append("precisao <" + str(C) + " e >" + str(C2) + ", tamanhos dos texto1 e 2 < 20")
                    portextocurto += 1
            else:
                #print "boa precisao! adicionou direto"
                g.AddGraphEdge(v0, v1)
                weights.InsertNextValue(weightListaux[k] / 1.0)
                weightList.append(weightListaux[k] / 1.0)
                colors_e.InsertNextValue(getColorEdge(weightListaux[k]))
                edgesList.append([v0, v1])
                motivoEdge.append("precisao < " + str(C2))
                porotimo += 1

    print "Calculou arestas de vizinhos e precisao direta"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    #Para cada vertice que nao foi ligado a outro por nao possuir grande similaridade
    #crie uma aresta com a melhor similaridade possivel; menor coeficiente possivel.
    aloneVList = []    
    aloneVList = vertex[:]    

    for e in edgesList:
        try:
            aloneVList.remove(e[0])
        except:
            try:
                aloneVList.remove(e[1])
            except:
                pass    

    #Retorna a aresta com menor coeficiente possivel para o vertice v0
    def getBetterSimilarityEdge(v0):
        better_w = 5000000
        better_v = 0
        #considerar apenas arestas de distancia menor que c
        c = getConstPerCent(5)
        #if(len(notAloneVList)> 0):
        for v1 in vertex:
            if v1 != v0:
                w = similarity(words_text[v0], words_text[v1])
                if w == -1 or w < 2:
                    w = max
                if w < better_w:
                    better_w = w
                    better_v = v1
        if better_w > c:
            return -1
        else:
            return better_v
        #else:
        #    return -1

    #Cria a aresta com menor coeficiente possivel
    for v0 in aloneVList:
        v1 = getBetterSimilarityEdge(v0)
        if v1 != -1 and v0 != v1 and [v0, v1] not in edgesList and [v1, v0] not in edgesList:
            g.AddGraphEdge(v0, v1)
            #O peso nestes casos nao importam e as arestas devem ser quase invisiveis
            weights.InsertNextValue(max)
            weightList.append(max)
            colors_e.InsertNextValue(getColorEdge(max))
            edgesList.append([v0, v1])
            motivoEdge.append("vertice sozinho")
            porsozinho += 1
            aloneVList.remove(v0)            

    print "Calculou arestas de vertices sozinhos"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    print "\nIniciando rendering..."
    # Add the edge weight array to the graph
    g.GetEdgeData().AddArray(weights)
    g.GetEdgeData().AddArray(colors_e)

    g.GetVertexData().AddArray(vertices)
    g.GetVertexData().AddArray(colors_v)

    graphLayoutView = vtk.vtkGraphLayoutView()

    if sys.argv[2] == "2D3D":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetInitialTemperature(0)
        #strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()
    elif sys.argv[2] == "2D":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetInitialTemperature(0)
        graphLayoutView.SetLayoutStrategy(strategy)
    elif sys.argv[2] == "5":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetInitialTemperature(5)
        strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()
    elif sys.argv[2] == "3D":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetThreeDimensionalLayout(1)
        strategy.SetInitialTemperature(0)
        strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()
    else:
        strategy = vtkForceDirectedLayoutStrategy()
        #strategy.SetThreeDimensionalLayout(1)
        strategy.SetInitialTemperature(5)
        strategy.SetAutomaticBoundsComputation(1)
        #strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()

    graphLayoutView.AddRepresentationFromInput(g)


    theme = vtkViewTheme.CreateMellowTheme()
    theme.SetPointSize(4)
    theme.SetLineWidth(1)
    theme.SetCellOpacity(.99)
    theme.SetSelectedCellColor(1, 0, 1)
    theme.SetSelectedPointColor(1, 0, 1)
    theme.SetBackgroundColor(0.2, 0.2, 0.2)
    theme.SetBackgroundColor2(0.1, 0.1, 0.1)
    graphLayoutView.ApplyViewTheme(theme)
    graphLayoutView.SetVertexLabelFontSize(4)
    theme.FastDelete()


    #graphLayoutView.SetLayoutStrategy("Simple 2D")
    graphLayoutView.SetEdgeLabelArrayName("Weights")
    #graphLayoutView.SetEdgeLabelVisibility(1)
    graphLayoutView.SetEdgeColorArrayName("color")

    graphLayoutView.SetColorEdges(True)

    graphLayoutView.SetVertexLabelArrayName("Vertices")
    #graphLayoutView.SetVertexLabelVisibility(1)
    graphLayoutView.SetVertexColorArrayName("color")
    graphLayoutView.SetColorVertices(True)

    #graphLayoutView.GetLayoutStrategy().SetEdgeWeightField("Weights")
    #graphLayoutView.GetLayoutStrategy().SetWeightEdges(1)

    #tempo.append(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    #print tempo
    graphLayoutView.ResetCamera()
    graphLayoutView.Render()

    print "Terminou rendering."
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    #graphLayoutView.GetLayoutStrategy().SetRandomSeed(0)

    #words_text_list[t]
    def getIntersectionTexts(index):
        #palavras que aparecem em todos os textos
        inter = words_text[index[0]]
        for t in index:
            inter = words_text[t] & inter
        return inter

    def selectionCallback(caller, event):
        # In C++ there is some extra data passed to the callback, but in Python
        # the callback data is lost...

        # There should be two selection nodes, but which one is vertices and which is edges
        # does not seem to be guaranteed...
        sel = caller.GetCurrentSelection()
        node0 = sel.GetNode(0)
        node0_field_type = node0.GetFieldType()
        sel_list0 = caller.GetCurrentSelection().GetNode(0).GetSelectionList()
        node1 = sel.GetNode(1)
        node1_field_type = node1.GetFieldType()
        sel_list1 = caller.GetCurrentSelection().GetNode(1).GetSelectionList()

        if (sel_list0.GetNumberOfTuples() > 0):
            printFieldType(node0_field_type)
            if node0_field_type == 3:
                if(sel_list0.GetNumberOfTuples() == 1):
                    x = sel_list0.GetValue(0)
                    edlist = [e for e in edgesList if weightList[edgesList.index(e)] < 2.2]
                    adjacents = [v[0] for v in edlist if v[1] == x]
                    adjacents = adjacents + [v[1] for v in edlist if v[0] == x]
                    adjacents = list(set(adjacents))
                    print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                    print "# Adjacents: #"
                    for nt in adjacents:
                        print "\t", titleList[nt] + " [" + classList[nt] + "]\n" + summaryList[nt] + "\n"
                    if(len(adjacents) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(adjacents)
                else:
                    index = []
                    for ii in range(sel_list0.GetNumberOfTuples()):
                        x = sel_list0.GetValue(ii)
                        print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                        index.append(x)
                    if(len(index) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(index)

            elif node0_field_type == 4:
                for ii in range(sel_list0.GetNumberOfTuples()):
                    x = sel_list0.GetValue(ii)
                    print "\t", str(motivoEdge[x])

        if (sel_list1.GetNumberOfTuples() > 0):
            printFieldType(node1_field_type)
            if node1_field_type == 3:
                if(sel_list1.GetNumberOfTuples() == 1):
                    x = sel_list1.GetValue(0)
                    edlist = [e for e in edgesList if weightList[edgesList.index(e)] < 2.2]
                    adjacents = [v[0] for v in edlist if v[1] == x]
                    adjacents = adjacents + [v[1] for v in edlist if v[0] == x]
                    adjacents = list(set(adjacents))
                    print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                    print "# Adjacents: #"
                    for nt in adjacents:
                        print "\t", titleList[nt] + " [" + classList[nt] + "]\n" + summaryList[nt] + "\n"
                    if(len(adjacents) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(adjacents)
                else:
                    index = []
                    for ii in range(sel_list1.GetNumberOfTuples()):
                        x = sel_list1.GetValue(ii)
                        print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                        index.append(x)
                    if(len(index) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(index)

            elif node1_field_type == 4:
                for ii in range(sel_list1.GetNumberOfTuples()):
                    x = sel_list1.GetValue(ii)
                    print "\t", str(motivoEdge[x])
        else:
            if(graphLayoutView.GetColorVertices()):
                graphLayoutView.SetColorVertices(False)
            else:
                graphLayoutView.SetColorVertices(True)


        print "- - -\n\n"
    #	GetAdjacentVertices (vtkIdType v, vtkAdjacentVertexIterator *it)
    def printFieldType(field_type):
        if field_type == 3:
            print "Vertices Selected:"
        elif field_type == 4:
            print "Edges Selected:"
        else:
            print "Unknown type:"

    rep = graphLayoutView.GetRepresentation(0)

    # The vtkRenderedGraphRepresentation should already have a vtkAnnotationLink,
    # so we just want to grab it and add an observer with our callback function attached
    link = rep.GetAnnotationLink()
    link.AddObserver("AnnotationChangedEvent", selectionCallback)

    #tempo.append(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    #print tempo
    print "Terminou"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
    print "sozinho: " + str(porsozinho)
    print "otimo: " + str(porotimo)
    print "texto curto: " + str(portextocurto)
    print "vizinhanca: " + str(porvizinho)
    graphLayoutView.GetInteractor().Start()
